package com.samp.solr.file2solr;

import com.google.common.hash.BloomFilter;
import com.google.common.hash.Funnels;
import org.apache.commons.lang3.StringUtils;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.params.CommonParams;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;


public class DataAnalyze {


    private static final String SOLR_FIELD_CONTENT = "content";
    private static final String SOLR_FIELD_INTENTION = "intention";
    private static final String SOLR_FIELD_CONTENT_FULL_MATCH = "contentFullMatch";
    private static final String BASE_SOLR_URL = "http://iqcd-ma66:8080/solr/gcc-omcp";
    private static final Integer QUERY_ROWS = 10;
    private static final String FLAG_YES = "1";
    private static final String FLAG_NO = "0";
    private static final Float MIN_RATE = 0.1f;

    private static int size = 40000000;
    private static BloomFilter<String> bloomFilter =
                                BloomFilter.create(Funnels.stringFunnel(Charset.forName("utf-8")), size, 0.0000001);


    private static void processData(String rawPath, String resultFilePath) throws Exception{
        Path filePath = Paths.get(rawPath);
        BufferedReader bufferReader = Files.newBufferedReader(filePath, StandardCharsets.UTF_8);
        String line;
        List<List<WordIntentionDTO>> totalResult = new ArrayList<>();
        try {
            int index = 1;
            SolrClient solrClient =new HttpSolrClient.Builder(BASE_SOLR_URL).build();
            while( (line = bufferReader.readLine()) != null ){
                //去重。
                String transText = transferText(line);
                if( bloomFilter.mightContain(transText) ){
//                    System.out.println("exist,skip:"+line);
                    continue;
                }
                bloomFilter.put(transText);
                String[] words = line.split("\t");
                if( words.length == 2 ){
                    List<WordIntentionDTO> result = querySolr(solrClient, words[0], words[1]);
                    totalResult.add(result);
                }else{
                    System.out.println("error line:"+line);
                    continue;
                }

                if( index % 1000 == 0 ){
                    System.out.println("process 1000 lines.");
                }
                index++;
            }
//            String[] words = line.split("\t");
//            List<WordIntentionDTO> result = querySolr(solrClient, words[0], words[1]);
//            totalResult.add(result);
            saveResultToFile(totalResult, resultFilePath);
            System.out.println("data process finished.");
        } catch (IOException e) {
            e.printStackTrace();
        }finally{
            bufferReader.close();
        }
        System.out.println("all done!");
    }

    /**
     * 转换文本，重新排序，也就是不考虑文本元素出现的顺序
     * @param text
     * @return
     */
    public static String transferText(String text){
        String[] words = text.split("\t");
        List<String> list = new ArrayList<>();
        for( String str : words[0].split(" ") ){
            list.add(str.trim());
        }
        Collections.sort(list);
        String intention = DataAnalyze.transferIntention(words[1]);
        if( intention == null ){
            return null;
        }
//        String intention = words[1];
        for( String str : intention.split(" ") ){
            list.add(str.trim());
        }
        return StringUtils.join(list.iterator(), " ");
    }

    private static void saveResultToFile(List<List<WordIntentionDTO>> totalResult, String resultFilePath) throws Exception{
        BufferedWriter writer = null;
        try {
            Integer index = 0;
            writer = Files.newBufferedWriter(Paths.get(resultFilePath), StandardCharsets.UTF_8);
            StringBuilder sb = new StringBuilder();
            writer.write("id\tqid1\tqid2\tquestion1\tquestion2\tis_duplicate\n");
            for( List<WordIntentionDTO> list: totalResult ){
                for( WordIntentionDTO dto: list ){
                    sb.append(index).append("\t").append(index).append("\t").append(index).append("\t");
                    sb.append(dto.getWord()).append("\t");
                    sb.append(transferIntention(dto.getIntention())).append("\t");
                    sb.append(dto.getFlag()).append("\n");
                    writer.write(sb.toString());
                    sb.setLength(0);
                    index++;
                }
            }
            writer.flush();
        } catch (IOException e1) {
            e1.printStackTrace();
        }finally{
            if( writer != null ){
                writer.close();
            }
        }

    }

    /**
     * 将action.name=查询#target.name=申请#target.params1=default#target.params2=default形式的意图转换成：
     * 查询 申请
     * @param rawIntention
     * @return
     */
    public static String transferIntention(String rawIntention){
        List<String> list = new ArrayList<>();
        String[] intentionSplits = rawIntention.trim().split("#");
        for( int i = 0 ;i < intentionSplits.length ; i++ ){
            String intention = intentionSplits[i];
            //去掉第四个槽位
            if( i == 3 ){
                continue;
            }
            String[] details = intention.split("=");
//            System.out.println(rawIntention);
            if( details.length < 2 ){
                return null;
            }
            if(  !"default".equals(details[1]) && !"execute".equals(details[1])){
                list.add(details[1]);
            }
        }
        return StringUtils.join(list.iterator(), " ");
    }

    private static List<WordIntentionDTO> querySolr(SolrClient solrClient, String word, String intention) throws Exception{
        SolrQuery query = new SolrQuery();
        query.set(CommonParams.QT, "/select");
        query.set(CommonParams.START, "0");
        query.set(CommonParams.ROWS, QUERY_ROWS);
        String queryParam = SOLR_FIELD_CONTENT+":"+word+" OR "+ SOLR_FIELD_CONTENT_FULL_MATCH+":"+"\""+word+"\"^8";
//        System.out.println(queryParam);
        query.set(CommonParams.Q, queryParam);
        query.set(CommonParams.FL, SOLR_FIELD_CONTENT+","+SOLR_FIELD_INTENTION+",score");
        List<WordIntentionDTO> resultList = new ArrayList<>();
        QueryResponse response = solrClient.query(query);
        SolrDocumentList documentList = response.getResults();
        //先循环一次，找出正确意图的score.
        Float rightScore = -1f;
        for( SolrDocument doc : documentList ){
            String docContent = (String)doc.getFieldValue(SOLR_FIELD_CONTENT);
            String docIntention = (String)doc.getFieldValue(SOLR_FIELD_INTENTION);
            if( word.equals(docContent) && intention.equals(docIntention) ){
                WordIntentionDTO dto = new WordIntentionDTO();
                dto.setWord(word);
                dto.setIntention(intention);
                dto.setFlag(FLAG_YES);
                resultList.add(dto);
                rightScore = (Float)doc.getFieldValue("score");
                break;
            }
        }
        if( rightScore > 0 ) {
            List<String> existList = new ArrayList<>();
            for (SolrDocument doc : documentList) {
//                String docContent = (String) doc.getFieldValue(SOLR_FIELD_CONTENT);
                String docIntention = (String) doc.getFieldValue(SOLR_FIELD_INTENTION);
                Float score = (Float) doc.getFieldValue("score");
                //如果意图已经有了，不再重复提交
                if( existList.contains(docIntention) ){
                    continue;
                }
                //意图一样的，剔除掉
                if( intention.equals(docIntention) ){
                    continue;
                }
                //分数比值小于指定值，也剔除掉
                Float rate = rightScore/score;
//                System.out.println("rate = "+rate);
                if( rate < MIN_RATE ){
                    continue;
                }
                WordIntentionDTO dto = new WordIntentionDTO();
                dto.setWord(word);
                dto.setIntention(docIntention);
                dto.setFlag(FLAG_NO);
                resultList.add(dto);
                existList.add(docIntention);
            }
        }else {
            System.out.println("solr未匹配正确结果：word="+word+", intention="+intention);
        }
        return resultList;
    }

    public static void main(String[] args) {
        String rawPath = "D:\\gcc-omcp\\temp\\base64_intention\\040102_final.txt";
        String outPath = "D:\\gcc-omcp\\temp\\base64_intention\\040102_out.txt";
        try {
            processData(rawPath, outPath);
        }catch (Exception e){
            e.printStackTrace();
        }
    }
}

class WordIntentionDTO{
    private String word;
    private String intention;
    private String flag;

    public String getWord() {
        return word;
    }

    public void setWord(String word) {
        this.word = word;
    }

    public String getIntention() {
        return intention;
    }

    public void setIntention(String intention) {
        this.intention = intention;
    }

    public String getFlag() {
        return flag;
    }

    public void setFlag(String flag) {
        this.flag = flag;
    }
}